In [1]:
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from keras.models import Sequential, model_from_json
from keras.layers import Dense, Dropout, Activation, Flatten, Convolution2D, MaxPooling2D
from keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
import numpy as np
import pandas as pd
from datetime import datetime
import matplotlib.pyplot as plt
%matplotlib inline
tf.logging.set_verbosity(tf.logging.ERROR)
In [2]:
tf.set_random_seed(2017)
np.random.seed(2017)
In [3]:
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# test data
test_images = mnist.test.images.reshape(10000, 28, 28, 1)
test_labels = mnist.test.labels[:]
augmentation_size = 440000
images = np.concatenate((mnist.train.images.reshape(55000, 28, 28, 1), mnist.validation.images.reshape(5000, 28, 28, 1)), axis=0)
labels = np.concatenate((mnist.train.labels, mnist.validation.labels), axis=0)
datagen_list = [
ImageDataGenerator(rotation_range=10),
ImageDataGenerator(rotation_range=20),
ImageDataGenerator(rotation_range=30),
ImageDataGenerator(width_shift_range=0.1),
ImageDataGenerator(width_shift_range=0.2),
ImageDataGenerator(width_shift_range=0.3),
]
for datagen in datagen_list:
datagen.fit(images)
for image, label in datagen.flow(images, labels, batch_size=augmentation_size, shuffle=True, seed=2017):
images = np.concatenate((images, image), axis=0)
labels = np.concatenate((labels, label), axis=0)
break
In [4]:
print('Train Data Set :', images.shape)
print('Test Data Set :', test_images.shape)
In [5]:
model1 = Sequential([Convolution2D(filters=64, kernel_size=(3, 3), padding='same', activation='elu', input_shape=(28, 28, 1)),
Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),
Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Convolution2D(filters=128, kernel_size=(3, 3), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Flatten(),
Dense(1024, activation='elu'),
Dropout(0.5),
Dense(1024, activation='elu'),
Dropout(0.5),
Dense(10, activation='softmax'),
])
model1.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])
model1.fit(images, labels, batch_size=256, epochs=20, shuffle=True, verbose=1, validation_data=(test_images, test_labels))
Out[5]:
In [6]:
model_json = model1.to_json()
with open("model1.json", "w") as json_file:
json_file.write(model_json)
model1.save_weights("model1.h5")
print("Saved model to disk")
In [7]:
model2 = Sequential([Convolution2D(filters=64, kernel_size=(5, 5), padding='same', activation='elu', input_shape=(28, 28, 1)),
Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),
Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Convolution2D(filters=128, kernel_size=(5, 5), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Flatten(),
Dense(1024, activation='elu'),
Dropout(0.5),
Dense(1024, activation='elu'),
Dropout(0.5),
Dense(10, activation='softmax'),
])
model2.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])
model2.fit(images, labels, batch_size=256, epochs=20, shuffle=True, verbose=1, validation_data=(test_images, test_labels))
Out[7]:
In [8]:
model_json = model2.to_json()
with open("model2.json", "w") as json_file:
json_file.write(model_json)
model2.save_weights("model2.h5")
print("Saved model to disk")
In [9]:
model3 = Sequential([Convolution2D(filters=64, kernel_size=(7, 7), padding='same', activation='elu', input_shape=(28, 28, 1)),
Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),
Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Convolution2D(filters=128, kernel_size=(7, 7), padding='same', activation='elu'),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.5),
Flatten(),
Dense(1024, activation='elu'),
Dropout(0.5),
Dense(1024, activation='elu'),
Dropout(0.5),
Dense(10, activation='softmax'),
])
model3.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])
model3.fit(images, labels, batch_size=256, epochs=20, shuffle=True, verbose=1, validation_data=(test_images, test_labels))
Out[9]:
In [10]:
model_json = model3.to_json()
with open("model3.json", "w") as json_file:
json_file.write(model_json)
model3.save_weights("model3.h5")
print("Saved model to disk")
In [11]:
from keras.models import model_from_json
from tensorflow.examples.tutorials.mnist import input_data
from keras.optimizers import Adam
import numpy as np
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
# test data
test_images = mnist.test.images.reshape(10000, 28, 28, 1)
test_labels = mnist.test.labels[:]
In [12]:
# load json and create model
def model_open(name, test_images, test_labels):
json_file = open(name + '.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights(name + '.h5')
print("Loaded model from disk")
loaded_model.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['acc'])
prob = loaded_model.predict_proba(test_images)
acc = np.mean(np.equal(np.argmax(prob, axis=1), np.argmax(test_labels, axis=1)))
print('\nmodel : %s, test accuracy : %.4f\n' % (name, acc))
return prob, acc
In [13]:
model_1_prob, model_1_acc = model_open('model1', test_images, test_labels)
model_2_prob, model_2_acc = model_open('model2', test_images, test_labels)
model_3_prob, model_3_acc = model_open('model3', test_images, test_labels)
In [14]:
def model_ensemble(prob1, acc1, prob2, acc2, prob3, acc3):
prob_list = [prob1, prob2, prob3]
acc_list = [acc1, acc2, acc3]
idx_acc_list = {idx: acc for idx, acc in enumerate(acc_list)}
sorted_acc_list = [idx for idx, _ in sorted(idx_acc_list.items(), key=lambda value: (value[1], value[0]), reverse=True)]
final_prob = 0
for i in sorted_acc_list:
final_prob += prob_list[i] * (i+1)
final_score = np.mean(np.equal(np.argmax(final_prob, axis=1), np.argmax(test_labels, axis=1))) # Test
print('Final test accuracy : %.4f' % final_score)
In [15]:
model_ensemble(model_1_prob, model_1_acc, model_2_prob, model_2_acc, model_3_prob, model_3_acc)